{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(100)\n",
    "x=np.random.normal(0, 1, (1200,200))\n",
    "y1=(np.random.normal(0, 1, 1200)>0).astype(int)\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "x_train, x_test, y_train11, y_test11 = train_test_split(x, y1, test_size=5/6)\n",
    "y_train1=np.hstack((y_train11[:,np.newaxis],1-y_train11[:,np.newaxis]))\n",
    "y_test1=np.hstack((y_test11[:,np.newaxis],1-y_test11[:,np.newaxis]))\n",
    " \n",
    "y_train21=(np.sum(x_train,axis=1)>0).astype(int)    \n",
    "y_train2=np.hstack((y_train21[:,np.newaxis],1-y_train21[:,np.newaxis]))\n",
    "y_test21=(np.sum(x_test,axis=1)>0).astype(int) \n",
    "y_test2=np.hstack((y_test21[:,np.newaxis],1-y_test21[:,np.newaxis]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 500000\n",
    "display_step = 50000\n",
    "\n",
    "# tf Graph Input\n",
    "X = tf.placeholder(tf.float32, [None, 200]) \n",
    "Y = tf.placeholder(tf.float32, [None, 2])\n",
    "\n",
    "# Set model weights\n",
    "W = tf.Variable(tf.random_normal([200, 2]))\n",
    "b = tf.Variable(tf.random_normal([2]))\n",
    "\n",
    "# Construct model\n",
    "pred = tf.nn.softmax(tf.matmul(X, W) + b) # Softmax\n",
    "\n",
    "# Minimize error using cross entropy\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=pred))\n",
    "# Gradient Descent\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "\n",
    "# Test model\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 500000\n",
    "display_step = 50000\n",
    "\n",
    "# tf Graph Input\n",
    "X = tf.placeholder(tf.float32, [None, 200]) \n",
    "Y = tf.placeholder(tf.float32, [None, 2])\n",
    "\n",
    "# Set model weights\n",
    "W = tf.Variable(tf.random_normal([200, 2]))\n",
    "b = tf.Variable(tf.random_normal([2]))\n",
    "\n",
    "# Construct model\n",
    "pred = tf.nn.softmax(tf.matmul(X, W) + b) # Softmax\n",
    "\n",
    "# Minimize error using cross entropy\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=pred))\n",
    "# Gradient Descent ADAM\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "\n",
    "# Test model\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "# Calculate accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    # Run the initializer\n",
    "    sess.run(init)\n",
    "\n",
    "    # Training cycle\n",
    "    for epoch in range(training_epochs):\n",
    "        _,c_train,acc_train = sess.run([optimizer,cost,accuracy], feed_dict={X: x_train,Y: y_train2})\n",
    "            # Display logs per epoch step\n",
    "        if(epoch+1) % display_step == 0:\n",
    "            print(\"train cost:\", c_train,\"train accuracy:\", acc_train)\n",
    "    print(\"Optimization Finished!\")  \n",
    "    print(\"test cost:\", cost.eval({X: x_test, Y: y_test2}),\"test Accuracy:\", accuracy.eval({X: x_test, Y: y_test2}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "def momentom(params,fs, lr,v,beta):\n",
    "    new_params=[]\n",
    "    grad= tf.gradients(xs=params, ys=fs) \n",
    "    grad, _ = tf.clip_by_global_norm(grad, 5)\n",
    "    for i,param in enumerate(params):\n",
    "        new_v=v[i].assign(beta*v[i]+lr*grad[i])\n",
    "        new_param=param.assign(param-new_v)\n",
    "        new_params.append([new_v,new_param])\n",
    "    return new_params\n",
    "# Parameters\n",
    "learning_rate = 0.01\n",
    "training_epochs = 200000\n",
    "batch_size = 200\n",
    "display_step = 20000\n",
    "beta=0.9\n",
    "\n",
    "# tf Graph Input\n",
    "X = tf.placeholder(tf.float32, [None, 200]) \n",
    "Y = tf.placeholder(tf.float32, [None, 2])\n",
    "\n",
    "# Set model weights\n",
    "W = tf.Variable(tf.random_normal([200, 2]))\n",
    "b = tf.Variable(tf.random_normal([2]))\n",
    "\n",
    "# Construct model\n",
    "pred = tf.nn.softmax(200*tf.matmul(X, W) + b) # Softmax\n",
    "\n",
    "# Minimize error using cross entropy\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=pred))\n",
    "# Gradient Descent\n",
    "# optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n",
    "params=[W,b]\n",
    "#Set momentum optimize variables\n",
    "V=[]\n",
    "for param in params:\n",
    "    V.append(tf.Variable(tf.zeros(param.get_shape())))\n",
    "# gradient update variables\n",
    "new_params=momentom(params,cost, learning_rate, V,beta)\n",
    "\n",
    "# Test model\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "# Calculate accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "    # Run the initializer\n",
    "    sess.run(init)\n",
    "    # Training cycle\n",
    "    for epoch in range(training_epochs):\n",
    "        _,c_train,acc_train = sess.run([new_params,cost,accuracy], feed_dict={X: x_train,Y: y_train2})\n",
    "            # Display logs per epoch step\n",
    "        if(epoch+1) % display_step == 0:\n",
    "            print(\"train cost:\", c_train,\"train accuracy:\", acc_train)  \n",
    "    print(\"test cost:\", cost.eval({X: x_test, Y: y_test2}),\"test Accuracy:\", accuracy.eval({X: x_test, Y: y_test2}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "lam_L=np.power(10,np.linspace(-4., 4., num=9))\n",
    "# Parameters\n",
    "learning_rate = 0.005\n",
    "training_epochs = 500000\n",
    "batch_size = 200\n",
    "display_step = 50000\n",
    "# tf Graph Input\n",
    "X = tf.placeholder(tf.float32, [None, 200]) \n",
    "Y = tf.placeholder(tf.float32, [None, 2])\n",
    "LAMDA = tf.placeholder(tf.float32)\n",
    "# Set model weights\n",
    "W = tf.Variable(tf.random_normal([200, 2]))\n",
    "b = tf.Variable(tf.random_normal([2]))\n",
    "\n",
    "# Construct model\n",
    "pred = tf.nn.softmax(tf.matmul(X, W) + b) # Softmax\n",
    "\n",
    "# Minimize error using cross entropy\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=pred))+LAMDA/2**tf.sqrt(tf.reduce_sum(W**2))/200\n",
    "\n",
    "# Gradient Descent\n",
    "# optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "# Test model\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "# Calculate accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()\n",
    "for lam in lam_L:\n",
    "    print(\"lam:\",lam)\n",
    "    # Start training\n",
    "    with tf.Session() as sess:\n",
    "        # Run the initializer \n",
    "        sess.run(init)\n",
    "\n",
    "        # Training cycle\n",
    "        for epoch in range(training_epochs):\n",
    "            _,c_train,acc_train = sess.run([optimizer,cost,accuracy], feed_dict={X: x_train,Y: y_train2,LAMDA:lam})\n",
    "            # Display logs per epoch step\n",
    "            if(epoch+1) % display_step == 0:\n",
    "                print(\"train cost:\", c_train,\"train accuracy:\", acc_train)\n",
    "        print(\"Optimization Finished!\")  \n",
    "        print(\"test cost:\", cost.eval({X: x_test, Y: y_test2,LAMDA:lam}),\"test Accuracy:\", accuracy.eval({X: x_test, Y: y_test2,LAMDA:lam}))\n",
    "        print(\"\\n\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
